import pandas as pd

# 改变路径为当前路径
import os

os.chdir(os.path.dirname(os.path.abspath(__file__)))

file_path = '攻略.xlsx'
df = pd.read_excel(file_path, engine='openpyxl')

specified_column = df['关键词1']

print(specified_column)

import jieba

word_list = [jieba.lcut(i) for i in specified_column if isinstance(i, str)]
print(word_list)

# 去除停用词
# import nltk
# nltk.download('stopwords')
# from nltk.corpus import stopwords
# stop_words = set(stopwords.words('chinese'))
# print(stop_words)

# 假设你已经有了一个中文停用词列表文件stopwords.txt
with open('stopwords.txt', 'r', encoding='utf-8') as f:
    stop_words = set([word.strip() for word in f.readlines()])

for i in range(len(word_list)):
    word_list[i] = [w for w in word_list[i] if not w in stop_words]

# 使用pyLDAvis进行可视化
import gensim
import gensim.corpora as corpora
from gensim.models import LdaModel
import pyLDAvis
import pyLDAvis.gensim_models as gensimvis
from pprint import pprint

# 创建词典
dictionary = corpora.Dictionary(word_list)
print("字典长度", len(dictionary), "字典内容", dictionary)
# 创建语料库
corpus = [dictionary.doc2bow(text) for text in word_list]
print("语料库长度", len(corpus), "语料库内容", corpus)

# 创建LDA模型
# lda_model = gensim.models.ldamulticore.LdaMulticore(corpus=corpus, num_topics=50, id2word=id2word, passes=30, workers=4)

# lda_model = gensim.models.ldamodel.LdaModel.load("lda_model")
# data = pyLDAvis.gensim_models.prepare(lda_model, corpus, id2word)
# pyLDAvis.display(data)
# pyLDAvis.save_html(data, "lda_model.html")

# 准备一个列表来保存每个主题数的困惑度和主题一致性
num_topics_range = range(2, 5)
perplexities = []
coherences = []

for num_topics in num_topics_range:
    # 训练LDA模型，并计算困惑度和主题一致性
    lda_model = LdaModel(corpus=corpus, id2word=dictionary, num_topics=num_topics, random_state=100, update_every=1,
                         chunksize=100, passes=10, alpha='auto', per_word_topics=True)

    perplexity = lda_model.log_perplexity(corpus)
    perplexities.append(perplexity)
    coherence = gensim.models.CoherenceModel(model=lda_model, texts=word_list, dictionary=dictionary,
                                             coherence='c_v').get_coherence()

    print(f"Num Topics: {num_topics}, Perplexity: {perplexity:.2f}, Coherence: {coherence:.4f}")
    from gensim.models import CoherenceModel

    coherence_model_lda = CoherenceModel(model=lda_model, texts=word_list, dictionary=dictionary, coherence='c_v')
    coherence_lda = coherence_model_lda.get_coherence()
    coherences.append(coherence_lda)
    print(f"Num Topics: {num_topics}, Perplexity: {perplexity:.2f}, Coherence: {coherence_lda:.4f}")